{
  "cells": [
    {
      "cell_type": "markdown",
      "id": "cell_0",
      "metadata": {},
      "source": [
        "# Advanced Usage Example - Extracting aspects and concepts from a document, with references, using concurrency"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "cell_1",
      "metadata": {},
      "outputs": [],
      "source": [
        "%pip install -U contextgem"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "cell_2",
      "metadata": {},
      "source": [
        "To run the extraction, please provide your LLM details in the ``DocumentLLM(...)`` constructor further below."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "cell_3",
      "metadata": {},
      "outputs": [],
      "source": [
        "# Advanced Usage Example - Extracting aspects and concepts from a document, with references,\n",
        "# using concurrency\n",
        "\n",
        "import os\n",
        "\n",
        "from aiolimiter import AsyncLimiter\n",
        "\n",
        "from contextgem import (\n",
        "    Aspect,\n",
        "    BooleanConcept,\n",
        "    DateConcept,\n",
        "    Document,\n",
        "    DocumentLLM,\n",
        "    JsonObjectConcept,\n",
        "    StringConcept,\n",
        ")\n",
        "\n",
        "# Example privacy policy document (shortened for brevity)\n",
        "doc = Document(\n",
        "    raw_text=(\n",
        "        \"Privacy Policy\\n\\n\"\n",
        "        \"Last Updated: March 15, 2024\\n\\n\"\n",
        "        \"1. Data Collection\\n\"\n",
        "        \"We collect various types of information from our users, including:\\n\"\n",
        "        \"- Personal information (name, email address, phone number)\\n\"\n",
        "        \"- Device information (IP address, browser type, operating system)\\n\"\n",
        "        \"- Usage data (pages visited, time spent on site)\\n\"\n",
        "        \"- Location data (with your consent)\\n\\n\"\n",
        "        \"2. Data Usage\\n\"\n",
        "        \"We use your information to:\\n\"\n",
        "        \"- Provide and improve our services\\n\"\n",
        "        \"- Send you marketing communications (if you opt-in)\\n\"\n",
        "        \"- Analyze website performance\\n\"\n",
        "        \"- Comply with legal obligations\\n\\n\"\n",
        "        \"3. Data Sharing\\n\"\n",
        "        \"We may share your information with:\\n\"\n",
        "        \"- Service providers (for processing payments and analytics)\\n\"\n",
        "        \"- Law enforcement (when legally required)\\n\"\n",
        "        \"- Business partners (with your explicit consent)\\n\\n\"\n",
        "        \"4. Data Retention\\n\"\n",
        "        \"We retain personal data for 24 months after your last interaction with our services. \"\n",
        "        \"Analytics data is kept for 36 months.\\n\\n\"\n",
        "        \"5. User Rights\\n\"\n",
        "        \"You have the right to:\\n\"\n",
        "        \"- Access your personal data\\n\"\n",
        "        \"- Request data deletion\\n\"\n",
        "        \"- Opt-out of marketing communications\\n\"\n",
        "        \"- Lodge a complaint with supervisory authorities\\n\\n\"\n",
        "        \"6. Contact Information\\n\"\n",
        "        \"For privacy-related inquiries, contact our Data Protection Officer at privacy@example.com\\n\"\n",
        "    ),\n",
        ")\n",
        "\n",
        "# Define all document-level concepts in a single declaration\n",
        "document_concepts = [\n",
        "    BooleanConcept(\n",
        "        name=\"Is Privacy Policy\",\n",
        "        description=\"Verify if this document is a privacy policy\",\n",
        "        singular_occurrence=True,  # explicitly enforce singular extracted item (optional)\n",
        "    ),\n",
        "    DateConcept(\n",
        "        name=\"Last Updated Date\",\n",
        "        description=\"The date when the privacy policy was last updated\",\n",
        "        singular_occurrence=True,  # explicitly enforce singular extracted item (optional)\n",
        "    ),\n",
        "    StringConcept(\n",
        "        name=\"Contact Information\",\n",
        "        description=\"Contact details for privacy-related inquiries\",\n",
        "        add_references=True,\n",
        "        reference_depth=\"sentences\",\n",
        "    ),\n",
        "]\n",
        "\n",
        "# Define all aspects with their concepts in a single declaration\n",
        "aspects = [\n",
        "    Aspect(\n",
        "        name=\"Data Collection\",\n",
        "        description=\"Information about what types of data are collected from users\",\n",
        "        concepts=[\n",
        "            JsonObjectConcept(\n",
        "                name=\"Collected Data Types\",\n",
        "                description=\"List of different types of data collected from users\",\n",
        "                structure={\n",
        "                    \"personal_info\": list[str],\n",
        "                    \"technical_info\": list[str],\n",
        "                    \"usage_info\": list[str],\n",
        "                },  # simply use a dictionary with type hints (including generic aliases and union types)\n",
        "                add_references=True,\n",
        "                reference_depth=\"sentences\",\n",
        "            )\n",
        "        ],\n",
        "    ),\n",
        "    Aspect(\n",
        "        name=\"Data Retention\",\n",
        "        description=\"Information about how long different types of data are retained\",\n",
        "        concepts=[\n",
        "            JsonObjectConcept(\n",
        "                name=\"Retention Periods\",\n",
        "                description=\"The durations for which different types of data are retained\",\n",
        "                structure={\n",
        "                    \"personal_info\": str | None,\n",
        "                    \"technical_info\": str | None,\n",
        "                    \"usage_info\": str | None,\n",
        "                },  # use `str | None` type hints to allow for None values if not specified\n",
        "                add_references=True,\n",
        "                reference_depth=\"sentences\",\n",
        "                singular_occurrence=True,  # explicitly enforce singular extracted item (optional)\n",
        "            )\n",
        "        ],\n",
        "    ),\n",
        "    Aspect(\n",
        "        name=\"Data Subject Rights\",\n",
        "        description=\"Information about the rights users have regarding their data\",\n",
        "        concepts=[\n",
        "            StringConcept(\n",
        "                name=\"Data Subject Rights\",\n",
        "                description=\"Rights available to users regarding their personal data\",\n",
        "                add_references=True,\n",
        "                reference_depth=\"sentences\",\n",
        "            )\n",
        "        ],\n",
        "    ),\n",
        "]\n",
        "\n",
        "# Add aspects and concepts to the document\n",
        "doc.add_aspects(aspects)\n",
        "doc.add_concepts(document_concepts)\n",
        "\n",
        "# Create an LLM for extraction\n",
        "llm = DocumentLLM(\n",
        "    model=\"openai/gpt-4o\",  # or another LLM from e.g. Anthropic, Ollama, etc.\n",
        "    api_key=os.environ.get(\n",
        "        \"CONTEXTGEM_OPENAI_API_KEY\"\n",
        "    ),  # your API key for the applicable LLM provider\n",
        "    async_limiter=AsyncLimiter(\n",
        "        3, 3\n",
        "    ),  # customize async limiter for concurrency (optional)\n",
        ")\n",
        "\n",
        "# Extract all information from the document, using concurrency\n",
        "doc = llm.extract_all(doc, use_concurrency=True)\n",
        "\n",
        "# Access / print extracted information on the document object\n",
        "\n",
        "print(\"Document Concepts:\")\n",
        "for concept in doc.concepts:\n",
        "    print(f\"{concept.name}:\")\n",
        "    for item in concept.extracted_items:\n",
        "        print(f\"\u2022 {item.value}\")\n",
        "    print()\n",
        "\n",
        "print(\"Aspects and Concepts:\")\n",
        "for aspect in doc.aspects:\n",
        "    print(f\"[{aspect.name}]\")\n",
        "    for item in aspect.extracted_items:\n",
        "        print(f\"\u2022 {item.value}\")\n",
        "    print()\n",
        "    for concept in aspect.concepts:\n",
        "        print(f\"{concept.name}:\")\n",
        "        for item in concept.extracted_items:\n",
        "            print(f\"\u2022 {item.value}\")\n",
        "    print()\n"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.0"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}